<!DOCTYPE html>
<html lang="zh-cn">
<head>
    <meta charset="utf-8"/>
    <meta name="viewport" content="width=device-width, initial-scale=1.0"/>
    <meta name="referrer" content="origin-when-cross-origin"/>
    <meta name="description"
          content=""/>
    <meta property="og:description"
          content=""/>
    <meta http-equiv="Cache-Control" content="no-transform"/>
    <meta http-equiv="Cache-Control" content="no-siteapp"/>
    <meta http-equiv="X-UA-Compatible" content="IE=edge"/>
    <title>【机器学习】数据准备--python爬虫</title>
    <link rel="stylesheet" href="/css/blog-common.min.css"/>

    <link type="text/css" rel="stylesheet" href="/css/cnblogs.css"/>
    <link type="text/css" rel="stylesheet" href="/css/custom.css"/>

    <link id="mobile-style" media="only screen and (max-width: 767px)" type="text/css" rel="stylesheet"
          href="/css/mobile-common.min.css"/>

    <script>
        var currentBlogId = 721026;
        var currentBlogApp = 'GuoXiongfei';
        var isLogined = true;
        var isBlogOwner = false;
        var skinName = 'Custom';
        var visitorUserId = '4410638e-090f-e011-ac81-842b2b196315';
        var hasCustomScript = true;
        try {
            if (hasCustomScript && document.referrer && document.referrer.indexOf('baidu.com') >= 0) {
                Object.defineProperty(document, 'referrer', {value: ''});
                Object.defineProperty(Document.prototype, 'referrer', {
                    get: function () {
                        return '';
                    }
                });
            }
        } catch (error) {
        }
        window.cb_enable_mathjax = true;
        window.mathEngine = 0;
        window.codeHighlightEngine = 1;
        window.enableCodeLineNumber = false;
        window.codeHighlightTheme = 'cnblogs';
        window.isDarkCodeHighlightTheme = false;
        window.isDisableCodeHighlighter = false;
    </script>
    <script>
        var currentPostDateAdded = '2022-06-22 21:38';
    </script>
    <script src="/js/jquery-2.2.0.min.js"></script>
    <script src="/js/blog-common.min.js?t=01011"></script>

</head>
<body class="skin-custom has-navbar mathjax2">
<a name="top"></a>
<div id="bannerbar" class="bannerbar-mobile formobile">
    <a href="https://developer.aliyun.com/trainingcamp/88b6444d8728476090ac3aa1338179f2?utm_content=g_1000340106"
       onclick="countCreativeClicks('M2-阿里云-数据库训练营')" rel="nofollow">
        <img src="https://img2022.cnblogs.com/blog/35695/202205/35695-20220510214350477-1439286895.png" alt=""
             onload="countCreativeImpressions('M2-阿里云-数据库训练营')"/>
    </a>
</div>
<div id="top_nav" class="navbar forpc">
    <nav id="nav_main" class="navbar-main">
        <ul id="nav_left" class="navbar-list navbar-left">
            <li>
                <a href="/">首页</a>
            </li>
            <li>
                <a href="https://www.guoxiongfei.cn/" onclick="countClicks('nav', 'skin-navbar-news')">新闻</a>
            </li>
            <li>
                <a href="http://www.guoxiongfei.cn/" onclick="countClicks('nav', 'skin-navbar-q')">博问</a>
            </li>
        </ul>
        <ul id="nav_right" class="navbar-list navbar-right">
            <li>
                <form id="zzk_search" class="navbar-search" action="http://www.guoxiongfei.cn/search.html?q="
                      method="get"
                      role="search">
                    <input name="w" id="zzk_search_input" placeholder="代码改变世界" type="search" tabindex="3"/>
                    <button type="submit" id="zzk_search_button">
                        <img src="/images/aggsite/search.svg" alt="搜索"/>
                    </button>
                </form>
            </li>
        </ul>
    </nav>
</div>

<div id="page_begin_html">
    <div id="loading">
        <div class="boxLoading"></div>
    </div>
</div>

<!--done-->
<div id="home">
    <div id="header">
        <div id="blogTitle">
            <a id="lnkBlogLogo" href="https://www.cnblogs.com/GuoXiongfei/"><img id="blogLogo"
                                                                              src="/skins/custom/images/logo.gif"
                                                                              alt="返回主页"/></a>

            <!--done-->
            <h1><a id="Header1_HeaderTitle" class="headermaintitle HeaderMainTitle"
                   href="https://www.cnblogs.com/GuoXiongfei/">GuoXiongfei</a>
            </h1>
            <h2></h2>


        </div><!--end: blogTitle 博客的标题和副标题 -->
        <div id="navigator">

            <ul id="navList">
                <li>
                    <a id="blog_nav_myhome" class="menu" href="https://www.guoxiongfei.cn">
                        首页</a>
                </li>
                <li>
                    <a id="blog_nav_contact" class="menu" href="https://www.guoxiongfei.cn/post.html">
                        联系</a></li>
                <li>
                    <a id="blog_nav_rss" class="menu" href="javascript:void(0)"
                       data-rss="https://www.guoxiongfei.cn/rss/">
                        订阅</a></li>
            </ul>


        </div><!--end: navigator 博客导航栏 -->
    </div><!--end: header 头部 -->
    <div id="main">
        <div id="mainContent">
            <div class="forFlow">
                <div id="post_detail">
                    <!--done-->
                    <div id="topics">
                        <div class="post">
                            <h1 class="postTitle">

                                <a id="cb_post_title_url" class="postTitle2 vertical-middle"
                                   href="https://www.cnblogs.com/GuoXiongfei/p/16395596.html">
                                    <span role="heading" aria-level="2">【机器学习】数据准备</span>

                                </a>

                            </h1>
                            <div class="clear"></div>
                            <div class="postBody">
                                <div id="cnblogs_post_body" class="blogpost-body cnblogs-markdown">
                                    <h2 id="前言">前言<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h2>
                                    <p>我们在学习机器学习相关内容时，一般是不需要我们自己去爬取数据的，因为很多的算法学习很友好的帮助我们打包好了相关数据，但是这并不代表我们不需要进行学习和了解相关知识。在这里我们了解三种数据的爬取：鲜花/明星图像的爬取、中国艺人图像的爬取、股票数据的爬取。分别对着三种爬虫进行学习和使用。</p>
                                    <ul>
                                        <li>体会<br>
                                            个人感觉爬虫的难点就是URL的获取，URL的获取与自身的经验有关，这点我也很难把握，一般URL获取是通过访问该网站通过抓包进行分析获取的。一般也不一定需要抓包工具，通过浏览器的开发者工具（F12/Fn+F12）即可进行获取。</li>
                                    </ul>
                                    <h2 id="鲜花明星图像爬取">鲜花/明星图像爬取<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h2>
                                    <h3 id="url获取">URL获取<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>百度搜索鲜花关键词，并打开开发者工具，点击NrtWork</li>
                                    </ul>
                                    <p><img src="https://img2022.cnblogs.com/blog/2659401/202206/2659401-20220622175849326-599645495.png" alt="image" loading="lazy" class="medium-zoom-image"></p>
                                    <ul>
                                        <li>
                                            <p>找到数据包进行分析,分析重要参数<br>
                                                <img src="https://img2022.cnblogs.com/blog/2659401/202206/2659401-20220622175908765-159207057.png" alt="image" loading="lazy" class="medium-zoom-image"></p>
                                            <ul>
                                                <li>pn 表示第几张图片加载</li>
                                                <li>rn 表示加载多少图片</li>
                                            </ul>
                                        </li>
                                        <li>
                                            <p>查看返回值进行分析，可以看到图片体制在ThumbURL中<br>
                                                <img src="https://img2022.cnblogs.com/blog/2659401/202206/2659401-20220622175940627-2052116548.png" alt="image" loading="lazy" class="medium-zoom-image"></p>
                                        </li>
                                    </ul>
                                    <h3 id="下载过程">下载过程<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>
                                            <p><a href="http://image.baidu.com/search/acjson?" target="_blank" rel="noopener">http://image.baidu.com/search/acjson?</a> 百度图片地址</p>
                                        </li>
                                        <li>
                                            <p>拼接tn   进行访问可以得到每个图片的URL，在返回数据的thumbURL中<br>
                                                <a href="https://image.baidu.com/search/acjson?tn=resultjson_com&amp;logid=10338332981203604364&amp;ipn=rj&amp;ct=201326592&amp;is=&amp;fp=result&amp;fr=&amp;word=%7B%7D&amp;queryWord=%7B%7D&amp;cl=2&amp;lm=-1&amp;ie=utf-8&amp;oe=utf-8&amp;adpicid=&amp;st=&amp;z=&amp;ic=&amp;hd=&amp;latest=&amp;copyright=&amp;s=&amp;se=&amp;tab=&amp;width=&amp;height=&amp;face=&amp;istype=&amp;qc=&amp;nc=1&amp;expermode=&amp;nojc=&amp;isAsync=&amp;pn=%7B%7D&amp;rn=30&amp;gsm=78&amp;1650241802208=%27.format(keyword,keyword,30*i)" target="_blank" rel="noopener">https://image.baidu.com/search/acjson?+tn</a></p>
                                        </li>
                                        <li>
                                            <p>进行分离图片的URL然后访问下载</p>
                                        </li>
                                    </ul>
                                    <h3 id="代码">代码<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li><li>7</li><li>8</li><li>9</li><li>10</li><li>11</li><li>12</li><li>13</li><li>14</li><li>15</li><li>16</li><li>17</li><li>18</li><li>19</li><li>20</li><li>21</li><li>22</li><li>23</li><li>24</li><li>25</li><li>26</li><li>27</li><li>28</li><li>29</li><li>30</li><li>31</li><li>32</li><li>33</li><li>34</li><li>35</li><li>36</li><li>37</li><li>38</li><li>39</li><li>40</li><li>41</li><li>42</li><li>43</li><li>44</li><li>45</li><li>46</li><li>47</li><li>48</li><li>49</li><li>50</li><li>51</li><li>52</li><li>53</li><li>54</li><li>55</li><li>56</li><li>57</li><li>58</li><li>59</li><li>60</li><li>61</li><li>62</li><li>63</li><li>64</li><li>65</li><li>66</li><li>67</li><li>68</li><li>69</li><li>70</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-python"><span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> urllib

<span class="hljs-keyword">class</span> <span class="hljs-title class_">GetImage</span>():
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__init__</span>(<span class="hljs-params">self,keyword=<span class="hljs-string">'鲜花'</span>,paginator=<span class="hljs-number">1</span></span>):
        self.url = <span class="hljs-string">'http://image.baidu.com/search/acjson?'</span>

        self.headers = {
            <span class="hljs-string">'User-Agent'</span>: <span class="hljs-string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/102.0.0.0 Safari/537.36'</span>
        }

        self.keyword = keyword
        self.paginator = paginator


    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_param</span>(<span class="hljs-params">self</span>):

        keyword = urllib.parse.quote(self.keyword)
        params = []

        <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>,self.paginator+<span class="hljs-number">1</span>):
            params.append(
               <span class="hljs-string">'tn=resultjson_com&amp;logid=10338332981203604364&amp;ipn=rj&amp;ct=201326592&amp;is=&amp;fp=result&amp;fr=&amp;word={}&amp;queryWord={}&amp;cl=2&amp;lm=-1&amp;ie=utf-8&amp;oe=utf-8&amp;adpicid=&amp;st=&amp;z=&amp;ic=&amp;hd=&amp;latest=&amp;copyright=&amp;s=&amp;se=&amp;tab=&amp;width=&amp;height=&amp;face=&amp;istype=&amp;qc=&amp;nc=1&amp;expermode=&amp;nojc=&amp;isAsync=&amp;pn={}&amp;rn=30&amp;gsm=78&amp;1650241802208='</span>.<span class="hljs-built_in">format</span>(keyword,keyword,<span class="hljs-number">30</span>*i)

            )
        <span class="hljs-keyword">return</span> params
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_urls</span>(<span class="hljs-params">self,params</span>):
        urls = []
        <span class="hljs-keyword">for</span> param <span class="hljs-keyword">in</span> params:
            urls.append(self.url+param)
        <span class="hljs-keyword">return</span> urls

    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_image_url</span>(<span class="hljs-params">self,urls</span>):
        image_url = []
        <span class="hljs-keyword">for</span> url <span class="hljs-keyword">in</span> urls:
            json_data = requests.get(url,headers = self.headers).json()
            json_data = json_data.get(<span class="hljs-string">'data'</span>)
            <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> json_data:
                <span class="hljs-keyword">if</span> i:
                    image_url.append(i.get(<span class="hljs-string">'thumbURL'</span>))
        <span class="hljs-keyword">return</span> image_url
    <span class="hljs-keyword">def</span> <span class="hljs-title function_">get_image</span>(<span class="hljs-params">self,image_url</span>):
        <span class="hljs-comment">##根据图片url，存入图片</span>
        file_name = os.path.join(<span class="hljs-string">""</span>, self.keyword)
        <span class="hljs-comment">#print(file_name)</span>
        <span class="hljs-keyword">if</span> <span class="hljs-keyword">not</span> os.path.exists(file_name):
            os.makedirs(file_name)

        <span class="hljs-keyword">for</span> index,url <span class="hljs-keyword">in</span> <span class="hljs-built_in">enumerate</span>(image_url,start=<span class="hljs-number">1</span>):
            <span class="hljs-keyword">with</span> <span class="hljs-built_in">open</span>(file_name+<span class="hljs-string">'/{}.jpg'</span>.<span class="hljs-built_in">format</span>(index),<span class="hljs-string">'wb'</span>) <span class="hljs-keyword">as</span> f:
                f.write(requests.get(url,headers=self.headers).content)

            <span class="hljs-keyword">if</span> index != <span class="hljs-number">0</span> <span class="hljs-keyword">and</span> index%<span class="hljs-number">30</span> == <span class="hljs-number">0</span>:
                <span class="hljs-built_in">print</span>(<span class="hljs-string">"第{}页下载完成"</span>.<span class="hljs-built_in">format</span>(index/<span class="hljs-number">30</span>))


    <span class="hljs-keyword">def</span> <span class="hljs-title function_">__call__</span>(<span class="hljs-params">self, *args, **kwargs</span>):
        params = self.get_param()
        urls = self.get_urls(params)
        image_url = self.get_image_url(urls)
        self.get_image(image_url=image_url)


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:
    spider = GetImage(<span class="hljs-string">'鲜花'</span>,<span class="hljs-number">3</span>)
    spider()



</code></pre>
                                    <h3 id="明星图像爬取">明星图像爬取<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>只需要把main函数里的关键字换一下就可以了，换成明星即可</li>
                                    </ul>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-java">
<span class="hljs-type">if</span> <span class="hljs-variable">__name__</span> <span class="hljs-operator">=</span>= <span class="hljs-string">'__main__'</span>:
    spider = GetImage(<span class="hljs-string">'明星'</span>,<span class="hljs-number">3</span>)
    spider()

</code></pre>
                                    <h3 id="其他主题">其他主题<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>同理的我们需要其他图片也可以换</li>
                                    </ul>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-java"><span class="hljs-type">if</span> <span class="hljs-variable">__name__</span> <span class="hljs-operator">=</span>= <span class="hljs-string">'__main__'</span>:
    spider = GetImage(<span class="hljs-string">'动漫'</span>,<span class="hljs-number">3</span>)
    spider()
</code></pre>
                                    <h2 id="艺人图像爬取">艺人图像爬取<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h2>
                                    <h3 id="方法一">方法一<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>我们可以使用上面的爬取图片的方式，把关键词换为中国艺人也可以爬取图片</li>
                                    </ul>
                                    <h3 id="方法二">方法二<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <ul>
                                        <li>显然上面的方式可以满足我们部分需求，我们如果需要爬取不同艺人那么上面的方式就不是那么好了。</li>
                                        <li>我们下载10个不同艺人的图片，然后用他们的名字命名图片名，再把他们存入picture文件内</li>
                                    </ul>
                                    <h3 id="代码-1">代码<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li><li>7</li><li>8</li><li>9</li><li>10</li><li>11</li><li>12</li><li>13</li><li>14</li><li>15</li><li>16</li><li>17</li><li>18</li><li>19</li><li>20</li><li>21</li><li>22</li><li>23</li><li>24</li><li>25</li><li>26</li><li>27</li><li>28</li><li>29</li><li>30</li><li>31</li><li>32</li><li>33</li><li>34</li><li>35</li><li>36</li><li>37</li><li>38</li><li>39</li><li>40</li><li>41</li><li>42</li><li>43</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-python"><span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> os
<span class="hljs-keyword">import</span> urllib

<span class="hljs-keyword">def</span> <span class="hljs-title function_">getPicinfo</span>(<span class="hljs-params">url</span>):
    headers = {
       <span class="hljs-string">'User-Agent'</span>: <span class="hljs-string">'Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0'</span>,

    }
    response = requests.get(url,headers)

    <span class="hljs-keyword">if</span> response.status_code == <span class="hljs-number">200</span>:
        <span class="hljs-keyword">return</span> response.text
    <span class="hljs-keyword">return</span> <span class="hljs-literal">None</span>


Download_dir = <span class="hljs-string">'picture'</span>
<span class="hljs-keyword">if</span> os.path.exists(Download_dir) == <span class="hljs-literal">False</span>:
    os.mkdir(Download_dir)


pn_num = <span class="hljs-number">1</span>
rn_num = <span class="hljs-number">10</span>

<span class="hljs-keyword">for</span> k <span class="hljs-keyword">in</span> <span class="hljs-built_in">range</span>(pn_num):
    url = <span class="hljs-string">"https://sp0.baidu.com/8aQDcjqpAAV3otqbppnN2DJv/api.php?resource_id=28266&amp;from_mid=500&amp;format=json&amp;ie=utf-8&amp;oe=utf-8&amp;query=%E4%B8%AD%E5%9B%BD%E8%89%BA%E4%BA%BA&amp;sort_key=&amp;sort_type=1&amp;stat0=&amp;stat1=&amp;stat2=&amp;stat3=&amp;pn="</span>+<span class="hljs-built_in">str</span>(pn_num)+<span class="hljs-string">"&amp;rn="</span>+<span class="hljs-built_in">str</span>(rn_num)+<span class="hljs-string">"&amp;_=1580457480665"</span>
    res = getPicinfo(url)
    json_str = json.loads(res)
    figs = json_str[<span class="hljs-string">'data'</span>][<span class="hljs-number">0</span>][<span class="hljs-string">'result'</span>]

    <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> figs:
        name = i[<span class="hljs-string">'ename'</span>]
        img_url = i[<span class="hljs-string">'pic_4n_78'</span>]
        img_res = requests.get(img_url)
        <span class="hljs-keyword">if</span> img_res.status_code == <span class="hljs-number">200</span>:
            ext_str_splits = img_res.headers[<span class="hljs-string">'Content-Type'</span>].split(<span class="hljs-string">'/'</span>)
            ext = ext_str_splits[-<span class="hljs-number">1</span>]
            fname = name+<span class="hljs-string">'.'</span>+ext
            <span class="hljs-built_in">open</span>(os.path.join(Download_dir,fname),<span class="hljs-string">'wb'</span>).write(img_res.content)

            <span class="hljs-built_in">print</span>(name,img_url,<span class="hljs-string">'saved'</span>)

</code></pre>
                                    <h2 id="股票数据爬取">股票数据爬取<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h2>
                                    <p>我们对<a href="http://quote.eastmoney.com/center/gridlist.html" target="_blank" rel="noopener">http://quote.eastmoney.com/center/gridlist.html</a> 内的股票数据进行爬取，并且把数据储存下来</p>
                                    <h3 id="爬取代码">爬取代码<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li><li>7</li><li>8</li><li>9</li><li>10</li><li>11</li><li>12</li><li>13</li><li>14</li><li>15</li><li>16</li><li>17</li><li>18</li><li>19</li><li>20</li><li>21</li><li>22</li><li>23</li><li>24</li><li>25</li><li>26</li><li>27</li><li>28</li><li>29</li><li>30</li><li>31</li><li>32</li><li>33</li><li>34</li><li>35</li><li>36</li><li>37</li><li>38</li><li>39</li><li>40</li><li>41</li><li>42</li><li>43</li><li>44</li><li>45</li><li>46</li><li>47</li><li>48</li><li>49</li><li>50</li><li>51</li><li>52</li><li>53</li><li>54</li><li>55</li><li>56</li><li>57</li><li>58</li><li>59</li><li>60</li><li>61</li><li>62</li><li>63</li><li>64</li><li>65</li><li>66</li><li>67</li><li>68</li><li>69</li><li>70</li><li>71</li><li>72</li><li>73</li><li>74</li><li>75</li><li>76</li><li>77</li><li>78</li><li>79</li><li>80</li><li>81</li><li>82</li><li>83</li><li>84</li><li>85</li><li>86</li><li>87</li><li>88</li><li>89</li><li>90</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-python"><span class="hljs-comment"># http://quote.eastmoney.com/center/gridlist.html</span>
<span class="hljs-keyword">import</span> requests
<span class="hljs-keyword">from</span> fake_useragent <span class="hljs-keyword">import</span> UserAgent
<span class="hljs-keyword">import</span> json
<span class="hljs-keyword">import</span> csv
<span class="hljs-keyword">import</span>  urllib.request <span class="hljs-keyword">as</span> r
<span class="hljs-keyword">import</span> threading

<span class="hljs-keyword">def</span> <span class="hljs-title function_">getHtml</span>(<span class="hljs-params">url</span>):
    r = requests.get(url, headers={
        <span class="hljs-string">'User-Agent'</span>: UserAgent().random,
    })
    r.encoding = r.apparent_encoding
    <span class="hljs-keyword">return</span> r.text


<span class="hljs-comment"># 爬取多少</span>
num = <span class="hljs-number">20</span>

stockUrl = <span class="hljs-string">'http://52.push2.eastmoney.com/api/qt/clist/get?cb=jQuery112409623798991171317_1654957180928&amp;pn=1&amp;pz=20&amp;po=1&amp;np=1&amp;ut=bd1d9ddb04089700cf9c27f6f7426281&amp;fltt=2&amp;invt=2&amp;wbp2u=|0|0|0|web&amp;fid=f3&amp;fs=m:0+t:80&amp;fields=f1,f2,f3,f4,f5,f6,f7,f8,f9,f10,f12,f13,f14,f15,f16,f17,f18,f20,f21,f23,f24,f25,f22,f11,f62,f128,f136,f115,f152&amp;_=1654957180938'</span>


<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:
    responseText = getHtml(stockUrl)
    jsonText = responseText.split(<span class="hljs-string">"("</span>)[<span class="hljs-number">1</span>].split(<span class="hljs-string">")"</span>)[<span class="hljs-number">0</span>];
    resJson = json.loads(jsonText)
    datas = resJson[<span class="hljs-string">'data'</span>][<span class="hljs-string">'diff'</span>]
    dataList = []
    <span class="hljs-keyword">for</span> data <span class="hljs-keyword">in</span> datas:

        row = [data[<span class="hljs-string">'f12'</span>],data[<span class="hljs-string">'f14'</span>]]
        dataList.append(row)

    <span class="hljs-built_in">print</span>(dataList)

    f = <span class="hljs-built_in">open</span>(<span class="hljs-string">'stock.csv'</span>, <span class="hljs-string">'w+'</span>, encoding=<span class="hljs-string">'utf-8'</span>, newline=<span class="hljs-string">""</span>)
    writer = csv.writer(f)
    writer.writerow((<span class="hljs-string">"代码"</span>,<span class="hljs-string">"名称"</span>))
    <span class="hljs-keyword">for</span> data <span class="hljs-keyword">in</span> dataList:
        writer.writerow((data[<span class="hljs-number">0</span>]+<span class="hljs-string">"\t"</span>,data[<span class="hljs-number">1</span>]+<span class="hljs-string">"\t"</span>))
    f.close()


<span class="hljs-keyword">def</span> <span class="hljs-title function_">getStockList</span>():
    stockList = []
    f = <span class="hljs-built_in">open</span>(<span class="hljs-string">'stock.csv'</span>, <span class="hljs-string">'r'</span>, encoding=<span class="hljs-string">'utf-8'</span>)
    f.seek(<span class="hljs-number">0</span>)
    reader = csv.reader(f)
    <span class="hljs-keyword">for</span> item <span class="hljs-keyword">in</span> reader:
        stockList.append(item)

    f.close()
    <span class="hljs-keyword">return</span> stockList

<span class="hljs-keyword">def</span> <span class="hljs-title function_">downloadFile</span>(<span class="hljs-params">url,filepath</span>):

    <span class="hljs-keyword">try</span>:
        r.urlretrieve(url,filepath)
    <span class="hljs-keyword">except</span> Exception <span class="hljs-keyword">as</span> e:
        <span class="hljs-built_in">print</span>(e)
    <span class="hljs-built_in">print</span>(filepath,<span class="hljs-string">"is downLoaded"</span>)
    <span class="hljs-keyword">pass</span>

sem = threading.Semaphore(<span class="hljs-number">1</span>)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">dowmloadFileSem</span>(<span class="hljs-params">url,filepath</span>):
    <span class="hljs-keyword">with</span> sem:
        downloadFile(url,filepath)

urlStart = <span class="hljs-string">'http://quotes.money.163.com/service/chddata.html?code='</span>
urlEnd = <span class="hljs-string">'&amp;end=20210221&amp;fields=TCLOSW;HIGH;TOPEN;LCLOSE;CHG;PCHG;VOTURNOVER;VATURNOVER'</span>

<span class="hljs-keyword">if</span> __name__ == <span class="hljs-string">'__main__'</span>:
    stockList = getStockList()
    stockList.pop(<span class="hljs-number">0</span>)
    <span class="hljs-built_in">print</span>(stockList)


    <span class="hljs-keyword">for</span> s <span class="hljs-keyword">in</span> stockList:
        scode = <span class="hljs-built_in">str</span>(s[<span class="hljs-number">0</span>].split(<span class="hljs-string">"\t"</span>)[<span class="hljs-number">0</span>])

        url = urlStart+(<span class="hljs-string">"0"</span> <span class="hljs-keyword">if</span> scode.startswith(<span class="hljs-string">'6'</span>) <span class="hljs-keyword">else</span> <span class="hljs-string">'1'</span>)+ scode + urlEnd

        <span class="hljs-built_in">print</span>(url)
        filepath = (<span class="hljs-built_in">str</span>(s[<span class="hljs-number">1</span>].split(<span class="hljs-string">"\t"</span>)[<span class="hljs-number">0</span>])+<span class="hljs-string">"_"</span>+scode)+<span class="hljs-string">".csv"</span>
        threading.Thread(target=dowmloadFileSem,args=(url,filepath)).start()




</code></pre>
                                    <h3 id="数据处理代码">数据处理代码<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h3>
                                    <p>有可能当时爬取的数据是脏数据，运行下面代码不一定能跑通，需要你自己处理数据还是其他方法</p>
                                    <pre class="highlighter-hljs" highlighted="true"><div class="copy-btns">复制代码</div><div class="cbt-traffic_light"><span></span><span></span><span></span></div><ul class="awes-linenumber"><li>1</li><li>2</li><li>3</li><li>4</li><li>5</li><li>6</li><li>7</li><li>8</li><li>9</li><li>10</li><li>11</li><li>12</li><li>13</li><li>14</li><li>15</li><li>16</li><li>17</li><li>18</li><li>19</li><li>20</li><li>21</li><li>22</li><li>23</li><li>24</li><li>25</li><li>26</li><li>27</li><li>28</li><li>29</li><li>30</li><li>31</li><li>32</li><li>33</li><li>34</li><li>35</li><li>36</li><li>37</li><li>38</li><li>39</li><li>40</li><li>41</li><li>42</li><li>43</li><li>44</li><li>45</li><li>46</li><li>47</li><li>48</li><li>49</li><li>50</li><li>51</li><li>52</li><li>53</li><li>54</li><li>55</li><li>56</li><li>57</li><li>58</li><li>59</li><li>60</li><li>61</li><li>62</li><li>63</li><li>64</li><li>65</li><li>66</li><li>67</li><li>68</li><li>69</li><li>70</li><li>71</li></ul><div class="awes-lang">er-hljs</div><code class="highlighter-hljs hljs language-python"><span class="hljs-comment">## 主要利用matplotlib进行图像绘制</span>

<span class="hljs-keyword">import</span> pandas <span class="hljs-keyword">as</span> pd
<span class="hljs-keyword">import</span> matplotlib.pyplot <span class="hljs-keyword">as</span> plt
<span class="hljs-keyword">import</span> csv
<span class="hljs-keyword">import</span> 股票数据爬取 <span class="hljs-keyword">as</span> gp

plt.rcParams[<span class="hljs-string">'font.sans-serif'</span>] = [<span class="hljs-string">'simhei'</span>] <span class="hljs-comment">#指定字体</span>
plt.rcParams[<span class="hljs-string">'axes.unicode_minus'</span>] = <span class="hljs-literal">False</span> <span class="hljs-comment">#显示-号</span>
plt.rcParams[<span class="hljs-string">'figure.dpi'</span>] = <span class="hljs-number">100</span> <span class="hljs-comment">#每英寸点数</span>

files = []

<span class="hljs-keyword">def</span> <span class="hljs-title function_">read_file</span>(<span class="hljs-params">file_name</span>):
    data = pd.read_csv(file_name,encoding=<span class="hljs-string">'gbk'</span>)
    col_name = data.columns.values
    <span class="hljs-keyword">return</span> data,col_name

<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_file_path</span>():
    stock_list = gp.getStockList()
    paths = []
    <span class="hljs-keyword">for</span> stock <span class="hljs-keyword">in</span> stock_list[<span class="hljs-number">1</span>:]:
        p = stock[<span class="hljs-number">1</span>].strip()+<span class="hljs-string">"_"</span>+stock[<span class="hljs-number">0</span>].strip()+<span class="hljs-string">".csv"</span>
        <span class="hljs-built_in">print</span>(p)
        data,_=read_file(p)
        <span class="hljs-keyword">if</span> <span class="hljs-built_in">len</span>(data)&gt;<span class="hljs-number">1</span>:
            files.append(p)
            <span class="hljs-built_in">print</span>(p)

get_file_path()
<span class="hljs-built_in">print</span>(files)

<span class="hljs-keyword">def</span> <span class="hljs-title function_">get_diff</span>(<span class="hljs-params">file_name</span>):
    data,col_name = read_file(file_name)
    index = <span class="hljs-built_in">len</span>(data[<span class="hljs-string">'日期'</span>])-<span class="hljs-number">1</span>
    sep = index//<span class="hljs-number">15</span>
    plt.figure(figsize=(<span class="hljs-number">15</span>,<span class="hljs-number">17</span>))

    x = data[<span class="hljs-string">'日期'</span>].values.tolist()
    x.reverse()
    xticks = <span class="hljs-built_in">list</span>(<span class="hljs-built_in">range</span>(<span class="hljs-number">0</span>,<span class="hljs-built_in">len</span>(x),sep))
    xlabels = [x[i] <span class="hljs-keyword">for</span> i <span class="hljs-keyword">in</span> xticks]
    xticks.append(<span class="hljs-built_in">len</span>(x))


    y1 = [<span class="hljs-built_in">float</span>(c) <span class="hljs-keyword">if</span> c!=<span class="hljs-string">'None'</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> data[<span class="hljs-string">'涨跌额'</span>].values.tolist()]
    y2 = [<span class="hljs-built_in">float</span>(c) <span class="hljs-keyword">if</span> c != <span class="hljs-string">'None'</span> <span class="hljs-keyword">else</span> <span class="hljs-number">0</span> <span class="hljs-keyword">for</span> c <span class="hljs-keyword">in</span> data[<span class="hljs-string">'涨跌幅'</span>].values.tolist()]

    y1.reverse()
    y2.reverse()

    ax1 = plt.subplot(<span class="hljs-number">211</span>)
    plt.plot(<span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>,<span class="hljs-built_in">len</span>(x)+<span class="hljs-number">1</span>),y1,c=<span class="hljs-string">'r'</span>)
    plt.title(<span class="hljs-string">'{}-涨跌额/涨跌幅'</span>.<span class="hljs-built_in">format</span>(file_name.split(<span class="hljs-string">'_'</span>)[<span class="hljs-number">0</span>]),fontsize = <span class="hljs-number">20</span>)
    ax1.set_xticks(xticks)
    ax1.set_xticklabels(xlabels,rotation = <span class="hljs-number">40</span>)
    plt.ylabel(<span class="hljs-string">'涨跌额'</span>)

    ax2 = plt.subplot(<span class="hljs-number">212</span>)
    plt.plot(<span class="hljs-built_in">range</span>(<span class="hljs-number">1</span>, <span class="hljs-built_in">len</span>(x) + <span class="hljs-number">1</span>), y1, c=<span class="hljs-string">'g'</span>)
    <span class="hljs-comment">#plt.title('{}-涨跌额/涨跌幅'.format(file_name.splir('_')[0]), fontsize=20)</span>
    ax2.set_xticks(xticks)
    ax2.set_xticklabels(xlabels, rotation=<span class="hljs-number">40</span>)
    plt.xlabel(<span class="hljs-string">'日期'</span>)
    plt.ylabel(<span class="hljs-string">'涨跌额'</span>)
    plt.show()


<span class="hljs-built_in">print</span>(<span class="hljs-built_in">len</span>(files))
<span class="hljs-keyword">for</span> file <span class="hljs-keyword">in</span> files:
    get_diff(file)
</code></pre>
                                    <h2 id="总结">总结<button class="cnblogs-toc-button" title="显示目录导航" aria-expanded="false"></button></h2>
                                    <p>上文描述了三个数据爬取的案例，不同的数据爬取需要我们对不同的URL进行获取，不同参数进行输入，URL如何组合、如何获取、这是数据爬取的难点，需要有一定的经验和基础。</p>

                                </div>

                                <div class="clear"></div>
                                <div id="blog_post_info_block" role="contentinfo">
                                    <div id="blog_post_info"></div>
                                    <div class="clear"></div>
                                    <div id="post_next_prev"></div>
                                </div>
                            </div>
                            <div class="postDesc">posted @
                                <span id="post-date">2022-06-22 21:38</span>&nbsp;
                                <a href="https://www.cnblogs.com/GuoXiongfei/">GuoXiongfei</a>&nbsp;
                                阅读(<span id="post_view_count">467</span>)&nbsp;
                            </div>
                        </div>


                    </div><!--end: topics 文章、评论容器-->
                </div>

            </div><!--end: forFlow -->
        </div><!--end: mainContent 主体内容容器-->
        <div id="sideBar">
            <div id="sideBarMain">
                <div id="blog-calendar" style="display:none"></div>
                <div id="leftcontentcontainer">
                    <div id="blog-sidecolumn"></div>
                </div>
            </div><!--end: sideBarMain -->
        </div><!--end: sideBar 侧边栏容器 -->
        <div class="clear"></div>
    </div><!--end: main -->
    <div class="clear"></div>
    <div id="footer">
        <!--done-->
        Copyright &copy; 2022 www.GuoXiongfei.cn
    </div><!--end: footer -->
</div><!--end: home 自定义的最大容器 -->


<div id="page_end_html">
    <script src="https://guangzan.gitee.io/awescnb/index.js"></script>
    <script>
        const opts = {
            // 默认启用皮肤 'reacg'
            // 在这里添加自定义配置
            // 当前为全部使用默认配置
            theme: {
                name: 'geek',
                //color:'LightPink',
                color: 'LightBlue',
                title: '我的博客',
                avatar: 'https://pic.cnblogs.com/avatar/2659401/20211129173446.png',
                headerBackground: 'https://images.cnblogs.com/cnblogs_com/blogs/721026/galleries/2118026/o_220303035636_00000037.jpg'
                // ...
            },
            signature: {
                enable: true,
                contents: [
                    "欢迎来到GuoXiongfei的博客",
                    "<b>微笑向阳，无畏悲伤</b>",
                ],
            },
            // 代码高亮
            highLight: {
                enable: true,
            },
            // 代码行号
            lineNumbers: {
                enable: false,
            },
            // github图标
            github: {
                enable: true,
                color: '#ffb3cc',
                url: 'https://www.github.com/',
            },
            // 图片灯箱
            imagebox: {
                enable: true,
            },
            links: [
                {
                    name: 'Github',
                    link: 'https://www.guoxiongfei.cn',
                },
            ],

        }
        $.awesCnb(opts)
    </script>
</div>

</body>

<script type="application/javascript">
    var rf = false;
    var clearMenu;

    function initMenu() {
        clearMenu = setInterval(function () {
            if (rf) {
                clearInterval(clearMenu);
            } else {
                var menuObj = $("#cnblog-nav a");
                if (menuObj.eq(0).text().indexOf("博客园") > 0) {
                    menuObj.eq(0).remove();
                }
                if (menuObj.eq(0).text().indexOf("首页") > 0) {
                    menuObj.eq(0).attr("href", "https://www.guoxiongfei.cn");
                    $("#left-side .logo a").attr("href", "https://www.guoxiongfei.cn").html("ICoding");
                    rf = true;


                    //处理其他menu
                    for (var i = 0; i < menuObj.length; i++) {
                        if (menuObj.eq(i).text().length > 0 && (menuObj.eq(i).text().indexOf("新随笔") > 0
                            || menuObj.eq(i).text().indexOf("草稿箱") > 0 || menuObj.eq(i).text().indexOf("管理") > 0)) {
                            menuObj.eq(i).remove();
                        }

                        if (menuObj.eq(i).text().indexOf("联系") > 0) {
                            menuObj.eq(i).attr("href", "https://www.guoxiongfei.cn/resume.html");
                        }
                        if (menuObj.eq(i).text().indexOf("订阅") > 0) {
                            menuObj.eq(i).attr("href", "http://www.guoxiongfei.cn/post.html");
                        }
                    }
                    //sideBarMain.account
                    $("#sideBarMain .account").remove();

                    //custom-toolbar
                    var customBar = $(".custom-toolbar .toolbar-item");
                    for (var i = 0; i < customBar.length; i++) {
                        if(customBar.eq(i).text().indexOf("收藏")>-1){
                            customBar.eq(i).remove();
                        }
                        if(customBar.eq(i).text().indexOf("收起")>-1){
                            customBar.eq(i).click().click();
                        }
                    }
                }
            }
        }, 10);

    };
    $(function () {
        if (!rf) {
            initMenu();
        }
    });
</script>
</html>